import scrapy
import json
from turorial.items import DouyuspierItem


class DouyuSpider(scrapy.Spider):
    name = 'douyu'
    allowed_domains = ['capi.douyucdn.cn']
    page = 0
    url = 'http://capi.douyucdn.cn/api/v1/getVerticalRoom?limit=20&offset='
    start_urls = [url + str(page)]

    def parse(self, response):
        # 将返回的数据格式由json转换为字典类型，并取出其中data字段下的内容
        rooms = json.loads(response.text)['data']

        # 对每一个房间的数据进行处理
        for room in rooms:
            # 创建item对象
            item = DouyuspierItem()
            item['room_id'] = room['room_id']
            item['room_name'] = room['room_name']
            item['vertical_src'] = room['vertical_src']

            yield item

        # 开始处理下一页数据
        self.page += 20
        yield scrapy.Request(self.url + str(self.page), callback=self.parse)